%{
AUTHOR: Felipe Arteaga

DATE:  2020--
-------------------------------------------------------------------------
PROJECT:
-------------------------------------------------------------------------
DESCRIPTION:

Genera estadísticas generales de SAE


=========================================================================
%}

clc;clear;close all;fclose('all');
feature('DefaultCharacterSet', 'UTF-8');

pcName=char(java.lang.System.getProperty('user.name'));
if(strcmp(pcName,'felipe'))
    % PC Felipe
    myDir='/Users/felipe/Dropbox/';
    projectDir='/Users/felipe/Dropbox/git/warnings/';
    addpath(genpath([myDir,'/myMatlabFunctions/']));
end

anhos=2016:2020;
cantAnhos=length(anhos);

anhosConMatr=2016:2019;
cantAnhosConMatr=length(anhosConMatr);

dirFigures=[projectDir,'Paper/figuresCL/other/';];
dirFiguresLatex='figuresCL/other/';
dirTables=[projectDir,'/Paper/tablesCL/appendix/'];




%% Load

% data postulantes (del compilado datos publicos)
D=cell(cantAnhos,1);
D_conMatr=cell(cantAnhosConMatr,1);

for t=1:cantAnhos
    anhoStr=sprintf('%i',anhos(t));
D{t}=readCsv([myDir,'/Mineduc/dataCompilada/1. SAE/_csv/compiladoPostulantes',anhoStr,'.csv'],'delimiter',',');

end
for t=1:cantAnhosConMatr
    anhoStr=sprintf('%i',anhosConMatr(t));
D_conMatr{t}=readCsv([myDir,'/Mineduc/dataCompilada/5. Compilado/_csv/compiladoPostulantes',anhoStr,'.csv'],'delimiter',',');
end

% data oferta regular  (del compilado datos publicos)

O=cell(size(anhos));


for t=1:cantAnhos
    anhoStr=sprintf('%i',anhos(t));
O{t}=readtable([myDir,'/Mineduc/dataCompilada/1. SAE/_csv/oferta_1_',anhoStr,'.csv'],'delimiter',',');

end

% Geo
geo=readtable([myDir,'/Mineduc/dataOutput/georef/compilacionInfoGeoref_wNewMarket.csv'],'delimiter',',');
geoName=readtable([myDir,'/Mineduc/dataOutput/georef/nombresNewMarket2019.csv'],'delimiter',',');


%% Define year of implementation

onlyEntryGrade=[-1 0 1 7 9];
regiones=kron((1:16)',ones(14,1));
grades=kron(ones(16,1),(-1:12)');
anhoSae=2019*ones(16*14,1);
anhoSae(regiones==13)=2020;
anhoSae(ismember(regiones,[1 4 6 10]))=2018;
anhoSae(regiones==12)=2017;
anhoSae(not(ismember(grades,onlyEntryGrade)))=anhoSae(not(ismember(grades,onlyEntryGrade)))+1;

ordenEntrada=anhoSae-2016;

policy=table;
policy.cod_reg=regiones;
policy.anhoSae=anhoSae;
policy.grado=grades;
policy.ordenEntrada=ordenEntrada;
clearvars anhoSae regiones ordenEntrada grades entryGrade

save([myDir,'/Mineduc/dataCompilada/policy'],'policy');
writetable(policy,[myDir,'/Mineduc/dataCompilada/policy.csv']); 



%% Calculate relative size of urban zones in SAE

% Load datos matricula
dirMatr=readtable([myDir,'/Mineduc/dataInput/dataSchoolEnrollment/directorioMatr2020.csv']);
dirMatr.edu=dirMatr.eduNormal+dirMatr.eduLenguaje;
dirMatr=outerjoin(dirMatr,geo,'keys',{'rbd'},'mergeKeys',true,'type','left','rightVariables',{'newMarket'});
dirMatr=stataCollapse('newMarket',dirMatr(not(isnan(dirMatr.newMarket)),:),'edu','sum','sort',true);
dirMatr=outerjoin(dirMatr,geoName,'keys',{'newMarket'},'mergeKeys',true,'type','left','rightVariables',{'nomNewMarket'});
dirMatr=sortrows(dirMatr,'edu','descend');


colors=linspecer(2);
dirMatr.nomNewMarket=categorical(dirMatr.nomNewMarket);
cantCities=20;
barh(dirMatr.edu(1:cantCities),'facecolor',colors(1,:),'edgecolor','none')
hold on
barh(dirMatr.edu(1:5),'facecolor',colors(2,:),'edgecolor','none')
yticks(1:cantCities)
yticklabels(dirMatr.nomNewMarket(1:cantCities))
xticks([1e5 10e5])
set(gca,'xgrid','on')
xlabel('PK-12th Enrollment')
easyExport('/Users/felipe/Dropbox/projects/segregacion/figures/citySizes','xTickThousands',true,'noaxischange',true,'width',350)


%% Calculate penetration of SAE in implementation years

% In terms of number of schools-grade offerd

% Load complete supply:
ofertaLastYear=readtable('/Users/felipe/Dropbox/Mineduc/modelacion/riesgo/data/2020/mineduc/oferta_jpal_2020-09-02_fixed.csv');
ofertaLastYear=stataCollapse({'rbd','cod_nivel'},ofertaLastYear,{'total_cupos'},{'sum'},'sort',true);

for t=1:cantAnhos
    a=anhos(t);
    oferta_aux=stataCollapse({'rbd','cod_nivel'},O{t},{'cupos_totales'},{'sum'},'sort',true);

    oferta_aux.esta=true(height(oferta_aux),1);
    ofertaLastYear=outerjoin(ofertaLastYear,oferta_aux,'keys',{'rbd','cod_nivel'},'mergeKeys',true,'type','left','rightVariables',{'esta'});
    fprintf('\nAnho %i  hay un %.2f %% de la oferta final\n',a,100*mean(ofertaLastYear.esta))
    ofertaLastYear.esta=[];
    
end

%% Tabla general de stats de SAE

cantVars=6;
cantSubgroups=6; % General + 4 years of implementation (2020 is not considered)
tabla=nan(cantVars,cantAnhos, cantSubgroups);

rowsLab={'Applicants','% placed in preference','$\quad$% with no placement','$\quad$% change of school not realized'}';


for s=1:cantSubgroups
for i=1:cantAnhos
    % Load apps 
    a=anhos(i);
    data=D{i};
    
    % Si es asignado por distancia lo considero sin asginacion
    data.sinAsign_1(data.distancia_1==1)=1;
    data.realDevuelto=data.devueltoAsign_1==1&data.asignadoEnPref_1==0;
    
    % Subgroupo
    if(s>1)
        
        data=data(ismember([data.cod_reg_1,data.grado_1],[policy.cod_reg(policy.ordenEntrada==s-1),policy.grado(policy.ordenEntrada==s-1)],'rows'),:);
    end
    
    
    if(height(data)>0)
    data.ones=ones(height(data),1);
    auxF=stataCollapse({'ones'},data(data.participa_1==1,:),{'asignadoEnPref_1','participa_1','asignadoEnPref_1','sinAsign_1','realDevuelto','cantPrefDeclaradas_1'},{'sum','sum','mean','mean','mean','mean'},'sort',true );
    
    
    tabla(:,i,s)=[a*ones(height(auxF),1),auxF.participa_1_sum,auxF.asignadoEnPref_1_mean,auxF.sinAsign_1_mean,auxF.realDevuelto_mean,auxF.cantPrefDeclaradas_1_mean]';
    end
    % Postulantes 
    % sin asignacion en pref
    % Largo listas
    
end
end

%%

asignadoEnPref=reshape(tabla(3,:,1:end),cantAnhos,cantSubgroups)';
newSinAsign=reshape(tabla(4,:,1:end),cantAnhos,cantSubgroups)';
newDevuelto=reshape(tabla(5,:,1:end),cantAnhos,cantSubgroups)';
largoLista=reshape(tabla(6,:,1:end),cantAnhos,cantSubgroups)';

opts=struct;
opts.title='';
%opts.label='';
opts.header=mat2cellstr(anhos,'wts',0);
colLabels={'Aggregate','Since 2016','Since 2017','Since 2018','Since 2019','Since 2020'}';
opts.primeraColumna=[colLabels;colLabels;colLabels;colLabels];
opts.withAdjust=true;
opts.panel={0,'A. Share placed in preference';6,'B. Share not placed in preference without continuation option';12,'C. Share not placed in preference with continuation option';18,'D. Mean of application list length'};
%opts.file='';
opts.title='Aggregate Placement Results for Stable Populations by Year';
opts.footnote='In Panel B ``Share not placed in preference without continuation option'''' also includes applicants that do not have a current school because they are entering the schooling system. ``Since 201X'''' represents the zones and grades where the new centralized system was implemented in 201X. Therefore, each row represents a stable population. The category ``Change of school not realized'''' reflects students that applied from a school that offers the next grade and were not assigned to any submitted preference. They kept the seat at their school of origin.';
opts.file=[dirTables,'generalStatsSAE'];
opts.label='tabGeneralStatsSAE';
%opts.alignment=repmat({'l'},1,5);

opts.filasFantasma=[1 7 13 19];
tabl=cell2latex([mat2cellstr([asignadoEnPref;newSinAsign;newDevuelto],'decimalprecision','%.3f');mat2cellstr(largoLista,'decimalprecision','%.1f')],'opts',opts);
compileLatex(tabl)
sdf
%% Outside option for not assigned
% 
% 	label define razonFueraDeSae_l ///
% 		1 "(1) Se queda en su EE anterior" ///
% 		2 "(2) Particular pagado" ///
% 		3 "(3) EE solo parvularia" ///
% 		4 "(4) Jardin infantil" ///
% 		5 "(5) Educación lenguaje" ///
% 		6 "(6) Educación adultos" ///
% 		7 "(7) Educación especial" ///
% 		8 "(8) Artístico" ///
% 		9 "(9) Hospitalario" ///
% 		10 "(10) Otro" ///
% 		11 "(11) No está en ninguna base" ///
% 		12 "(12) Otro: privado, jardin o nn (actulizar base)" ///
% 		13 "(13) Otro: Jardin o nn (actulizar base parv)" ///
% 		19 "(19) EE en region sin SAE" 
% 	* La opción 12 es para cuando no estoy 

% 	label define descOrigen_l ///
% 		0 "(0) EE SAE con continuidad" ///
% 		1 "(1) EE SAE sin continuidad" ///
% 		2 "(2) Particular pagado" ///
% 		3 "(3) EE solo parvularia" ///
% 		4 "(4) Junji o Integra" ///
% 		5 "(5) Educación lenguaje" ///
% 		6 "(6) Educación adultos" ///
% 		7 "(7) Educación especial" ///
% 		8 "(8) Artístico" ///
% 		9 "(9) Hospitalario" ///
% 		10 "(10) Otro" ///
% 		11 "(11) No está en ninguna base" ///
% 		19 "(19) EE en region sin SAE"
        
% O enSAE
% 1 voucher outside SAE
% 2 private non-voucher
% 3 not observed
labelClasif={'School in SAE','Voucher outside SAE','Private non-voucher','Not observed'};

clasifAll=zeros(0,2);
for t=1:cantAnhosConMatr
    a=anhosConMatr(t);
    D_a=D_conMatr{t};
    
%tab(D_a.EEAnhoSigFueraDeSae(D_a.asignadoEnPref_1==0))
%tab(D_a.razonFueraDeSae(D_a.EEAnhoSigFueraDeSae==1&D_a.asignadoEnPref_1==0))

%tab(D_a.razonFueraDeSae(D_a.asignadoEnPref_1==0&D_a.devueltoAsign_1==0))

private=D_a.razonFueraDeSae==2|(D_a.razonFueraDeSae==1&D_a.descOrigen==2);
notObs=D_a.razonFueraDeSae==11;
enSAE=isnan(D_a.razonFueraDeSae);

sirven=D_a.asignadoEnPref_1==0&D_a.devueltoAsign_1==0&D_a.participa_1==1;

clasif=ones(height(D_a),1);
clasif(private)=2;
clasif(enSAE)=0;
clasif(notObs)=3;
clasif=clasif(sirven);


clasifAll=[clasifAll;[clasif,a*ones(size(clasif))]];

end
tabClasif=array2table(clasifAll,'VariableNames',{'clasif','anho'});
tabClasif.clasif=categorical(tabClasif.clasif,0:3,labelClasif);
tabClasif.Properties.VariableDescriptions{'anho'}='Year';
tabClasif.Properties.VariableDescriptions{'clasif'}='Classification of not assigned';
figure
bartab(tabClasif,'anho','clasif','includeall',0,'minpercentagetoshow',.03)
easyExport([dirFigures,'outsideOption'],'height',350,'erp',dirFiguresLatex)

%% What do devueltos do?
% 
% 	label define razonFueraDeSae_l ///
% 		1 "(1) Se queda en su EE anterior" ///
% 		2 "(2) Particular pagado" ///
% 		3 "(3) EE solo parvularia" ///
% 		4 "(4) Jardin infantil" ///
% 		5 "(5) Educación lenguaje" ///
% 		6 "(6) Educación adultos" ///
% 		7 "(7) Educación especial" ///
% 		8 "(8) Artístico" ///
% 		9 "(9) Hospitalario" ///
% 		10 "(10) Otro" ///
% 		11 "(11) No está en ninguna base" ///
% 		12 "(12) Otro: privado, jardin o nn (actulizar base)" ///
% 		13 "(13) Otro: Jardin o nn (actulizar base parv)" ///
% 		19 "(19) EE en region sin SAE" 
% 	* La opción 12 es para cuando no estoy 

% 	label define descOrigen_l ///
% 		0 "(0) EE SAE con continuidad" ///
% 		1 "(1) EE SAE sin continuidad" ///
% 		2 "(2) Particular pagado" ///
% 		3 "(3) EE solo parvularia" ///
% 		4 "(4) Junji o Integra" ///
% 		5 "(5) Educación lenguaje" ///
% 		6 "(6) Educación adultos" ///
% 		7 "(7) Educación especial" ///
% 		8 "(8) Artístico" ///
% 		9 "(9) Hospitalario" ///
% 		10 "(10) Otro" ///
% 		11 "(11) No está en ninguna base" ///
% 		19 "(19) EE en region sin SAE"
 
% -1 En devuelto
% O enSAE
% 1 voucher outside SAE
% 2 private non-voucher
% 3 not observed
labelClasif={'Original school','School in SAE','Voucher outside SAE','Private non-voucher','Not observed'};

clasifAll=zeros(0,2);
for t=1:cantAnhosConMatr
    a=anhosConMatr(t);
    D_a=D_conMatr{t};
    
%tab(D_a.EEAnhoSigFueraDeSae(D_a.asignadoEnPref_1==0))
%tab(D_a.razonFueraDeSae(D_a.EEAnhoSigFueraDeSae==1&D_a.asignadoEnPref_1==0))

%tab(D_a.razonFueraDeSae(D_a.asignadoEnPref_1==0&D_a.devueltoAsign_1==0))

private=D_a.razonFueraDeSae==2|(D_a.razonFueraDeSae==1&D_a.descOrigen==2);
notObs=D_a.razonFueraDeSae==11;
enSAE=isnan(D_a.razonFueraDeSae);
enDevuelto=D_a.rbdAsign_1==D_a.rbdAnhoSig;


sirven=D_a.asignadoEnPref_1==0&D_a.devueltoAsign_1==1&D_a.participa_1==1;

clasif=ones(height(D_a),1);
clasif(private)=2;
clasif(enSAE)=0;
clasif(enDevuelto)=-1;
clasif(notObs)=3;
clasif=clasif(sirven);

clasifAll=[clasifAll;[clasif,a*ones(size(clasif))]];

end
tabClasif=array2table(clasifAll,'VariableNames',{'clasif','anho'});
tabClasif.clasif=categorical(tabClasif.clasif,-1:3,labelClasif);
tabClasif.Properties.VariableDescriptions{'anho'}='Year';
tabClasif.Properties.VariableDescriptions{'clasif'}='Classification of frustrated changes';
figure
bartab(tabClasif,'anho','clasif','includeall',0,'minpercentagetoshow',.03)
easyExport([dirFigures,'devueltosDecision'],'height',350,'erp',dirFiguresLatex)


%% Plot implementation map

chile=shaperead([myDir,'/Mineduc/dataInput/shapefilesChile/cl_regiones_geo']);
preRegiones={chile.ID_2002};
S = sprintf('%s*', preRegiones{:});
regiones = sscanf(S, '%f*');

for r=1:length(regiones)
lat = extractfield(chile(regiones==regiones(r)), 'Y')';
lon = extractfield(chile(regiones==regiones(r)), 'X')';
[latreduced, lonreduced] = reducem(lat, lon,.08);

[chile(regiones==regiones(r)).X]=lonreduced;
[chile(regiones==regiones(r)).Y]=latreduced;

end


regionesPolicyA={12,[1 4 6 10 12],[1 2 3 4 5 6 7 8 9 10 11 12 14 15],1:15,1:15};
regionesPolicyB={[],12        ,[1 4 6 10 12]            ,[1 2 3 4 5 6 7 8 9 10 11 12 14 15],1:15};
anhosPolicy=[2016:2020];

colorA=[153,216,201]/256;
colorB=[44,162,95]/256*.8;
alpha=0;
tiledlayout(1,5);
for i=1:5
    
   
    nexttile
    
    hold on
    mapshow(chile,'FaceColor','none');
    mapshow(chile((ismember(regiones,regionesPolicyA{i}))),'FaceColor',colorA*(1-alpha)+[1 1 1]*alpha);
    mapshow(chile((ismember(regiones,regionesPolicyB{i}))),'FaceColor',colorB*(1-alpha)+[1 1 1]*alpha);
    
    text(-70,-58,sprintf('%i',anhosPolicy(i)),...
        'HorizontalAlignment','center','VerticalAlignment','top',...
        'FontSize',14,'Interpreter','latex')
    axis('off')
    
end
easyExport([dirFigures,'/implementationMap'],'height',300,'erp',dirFiguresLatex)

%% Tabla resumen participantes - warnings - surveys 2016-2020

postulantes=[3500,75000,275000,480000,455000]; % Primera Ronda
estab=[60,2000,6500,8000,8000];
si='\checkmark';
no='';
livefeedbak={no,si,si,si,si};
sms={si,si,si,si,si};
wa={no,no,no,no,si};
hs={si,si,no,no,no};
ts={no,si,si,no,no};
ms={no,no,no,no,si};


paper={no,no,si,si,si};

cellImprimir=[mat2cellstr([estab;postulantes]);sms;livefeedbak;wa;hs;ts;ms;paper];

column={'Schools','Applicants','SMS','In-platform pop-up (live)','WhatsApp','Household','Telephone','E-Mail','This paper'}';

opts=struct;
opts.title='';
opts.label='';
opts.header=mat2cellstr(anhosPolicy,'wts',0);
opts.primeraColumna=column;
opts.panel={0,'Participants of centralized admission system:';2,'Risk feedback strategy:';5,'Surveys'};
opts.filasFantasma=[8];
%opts.alignmentFirstCol={'L{3cm}'};
%opts.file='';

tabla=cell2latex(cellImprimir,'opts',opts);
compileLatex(tabla)

